# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

from scrapy.conf import settings
from scrapy.exceptions import DropItem
from pymongo import MongoClient


class MongoDBPipeline(object):

    def __init__(self):
        self.url = "mongodb://%(user)s:%(pwd)s@%(host)s:%(port)s/%(db)s" % {
            "user": settings.get("MONGODB_USER"),
            "pwd": settings.get("MONGODB_PWD"),
            "host": settings.get("MONGODB_SERVER"),
            "port": settings.get("MONGODB_PORT"),
            "db": settings.get("MONGODB_DB")
        }
        self.db = settings.get("MONGODB_DB")
        self.collection = settings.get('MONGODB_COLLECTION')

    def open_spider(self, spider):
        self.client = MongoClient(self.url)
        self.doc = self.client.get_database(self.db)[self.collection]

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        if self.doc.find({'issue':item.get('issue')}).count() > 0:
            raise DropItem("Duplicate item found:%s" % item.get('issue'))
        self.doc.insert(dict(item))
        return item